// HTMLParser Library $Name: v1_6 $ - A java-based parser for HTML
// http://sourceforge.org/projects/htmlparser
// Copyright (C) 2004 Derrick Oswald
//
// Revision Control Information
//
// $Source: /cvsroot/htmlparser/htmlparser/src/org/htmlparser/http/ConnectionManager.java,v $
// $Author: derrickoswald $
// $Date: 2006/06/02 02:43:24 $
// $Revision: 1.13 $
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//

package org.htmlparser.http;

import java.io.File;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.net.UnknownHostException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.Properties;
import java.util.StringTokenizer;
import java.util.Vector;

import org.htmlparser.util.ParserException;

/**
 * Handles proxies, password protected URLs and request properties including
 * cookies.
 */
public class ConnectionManager
{
	/**
	 * Default Request header fields. So far this is just "User-Agent" and
	 * "Accept-Encoding".
	 */
	protected static Hashtable mDefaultRequestProperties = new Hashtable();
	static
	{
		mDefaultRequestProperties.put("User-Agent", "HTMLParser/"
				+ org.htmlparser.lexer.Lexer.VERSION_NUMBER);
		mDefaultRequestProperties.put("Accept-Encoding", "gzip, deflate");
	}

	/**
	 * Messages for page not there (404).
	 */
	private static final String[] FOUR_OH_FOUR = {
			"The web site you seek cannot be located," + " but countless more exist",
			"You step in the stream, but the water has moved on." + " This page is not here.",
			"Yesterday the page existed. Today it does not." + " The internet is like that.",
			"That page was so big. It might have been very useful." + " But now it is gone.",
			"Three things are certain: death, taxes and broken links."
					+ " Guess which has occured.",
			"Chaos reigns within. Reflect, repent and enter the correct URL."
					+ " Order shall return.",
			"Stay the patient course. Of little worth is your ire." + " The page is not found.",
			"A non-existant URL reduces your expensive computer to a simple stone.",
			"Many people have visited that page." + " Today, you are not one of the lucky ones.",
			"Cutting the wind with a knife. Bookmarking a URL." + " Both are ephemeral.", };

	/**
	 * Base 64 character translation table.
	 */
	private static final char[] BASE64_CHAR_TABLE = ("ABCDEFGHIJKLMNOPQRSTUVWXYZ"
			+ "abcdefghijklmnopqrstuvwxyz0123456789+/").toCharArray();

	/**
	 * Request header fields.
	 */
	protected Hashtable mRequestProperties;

	/**
	 * The proxy server name.
	 */
	protected String mProxyHost;

	/**
	 * The proxy port number.
	 */
	protected int mProxyPort;

	/**
	 * The proxy username name.
	 */
	protected String mProxyUser;

	/**
	 * The proxy user password.
	 */
	protected String mProxyPassword;

	/**
	 * The username name for accessing the URL.
	 */
	protected String mUser;

	/**
	 * The user password for accessing the URL.
	 */
	protected String mPassword;

	/**
	 * Cookie storage, a hashtable (by site or host) of vectors of Cookies. This
	 * will be null if cookie processing is disabled (default).
	 */
	protected Hashtable mCookieJar;

	/**
	 * The object to be notified prior to and after each connection.
	 */
	protected ConnectionMonitor mMonitor;

	/**
	 * Flag determining if redirection processing is being handled manually.
	 */
	protected boolean mRedirectionProcessingEnabled;

	/**
	 * Cookie expiry date format for parsing.
	 */
	static protected SimpleDateFormat mFormat = new SimpleDateFormat("EEE, dd-MMM-yy kk:mm:ss z");

	/**
	 * Create a connection manager.
	 */
	public ConnectionManager()
	{
		this(getDefaultRequestProperties());
	}

	/**
	 * Create a connection manager with the given connection properties.
	 * 
	 * @param properties
	 *            Name/value pairs to be added to the HTTP request.
	 */
	public ConnectionManager(Hashtable properties)
	{
		mRequestProperties = properties;
		mProxyHost = null;
		mProxyPort = 0;
		mProxyUser = null;
		mProxyPassword = null;
		mUser = null;
		mPassword = null;
		mCookieJar = null;
		mMonitor = null;
		mRedirectionProcessingEnabled = false;
	}

	//
	// static methods
	//

	/**
	 * Get the current default request header properties. A String-to-String map
	 * of header keys and values. These fields are set by the parser when
	 * creating a connection.
	 * 
	 * @return The default set of request header properties that will currently
	 *         be used.
	 * @see #mDefaultRequestProperties
	 * @see #setRequestProperties
	 */
	public static Hashtable getDefaultRequestProperties()
	{
		return (mDefaultRequestProperties);
	}

	/**
	 * Set the default request header properties. A String-to-String map of
	 * header keys and values. These fields are set by the parser when creating
	 * a connection. Some of these can be set directly on a
	 * <code>URLConnection</code>, i.e. If-Modified-Since is set with
	 * setIfModifiedSince(long), but since the parser transparently opens the
	 * connection on behalf of the developer, these properties are not available
	 * before the connection is fetched. Setting these request header fields
	 * affects all subsequent connections opened by the parser. For more direct
	 * control create a <code>URLConnection</code> massage it the way you want
	 * and then set it on the parser.
	 * <p>
	 * From <a href="http://www.ietf.org/rfc/rfc2616.txt"> RFC 2616 Hypertext
	 * Transfer Protocol -- HTTP/1.1</a>:
	 * 
	 * <pre>
	 * 5.3 Request Header Fields
	 * 
	 *    The request-header fields allow the client to pass additional
	 *    information about the request, and about the client itself, to the
	 *    server. These fields act as request modifiers, with semantics
	 *    equivalent to the parameters on a programming language method
	 *    invocation.
	 * 
	 *        request-header = Accept                   ; Section 14.1
	 *                       | Accept-Charset           ; Section 14.2
	 *                       | Accept-Encoding          ; Section 14.3
	 *                       | Accept-Language          ; Section 14.4
	 *                       | Authorization            ; Section 14.8
	 *                       | Expect                   ; Section 14.20
	 *                       | From                     ; Section 14.22
	 *                       | Host                     ; Section 14.23
	 *                       | If-Match                 ; Section 14.24
	 *                       | If-Modified-Since        ; Section 14.25
	 *                       | If-None-Match            ; Section 14.26
	 *                       | If-Range                 ; Section 14.27
	 *                       | If-Unmodified-Since      ; Section 14.28
	 *                       | Max-Forwards             ; Section 14.31
	 *                       | Proxy-Authorization      ; Section 14.34
	 *                       | Range                    ; Section 14.35
	 *                       | Referer                  ; Section 14.36
	 *                       | TE                       ; Section 14.39
	 *                       | User-Agent               ; Section 14.43
	 * 
	 *    Request-header field names can be extended reliably only in
	 *    combination with a change in the protocol version. However, new or
	 *    experimental header fields MAY be given the semantics of request-
	 *    header fields if all parties in the communication recognize them to
	 *    be request-header fields. Unrecognized header fields are treated as
	 *    entity-header fields.
	 * </pre>
	 * 
	 * @param properties
	 *            The new set of default request header properties to use. This
	 *            affects all subsequently created connections.
	 * @see #mDefaultRequestProperties
	 * @see #setRequestProperties
	 */
	public static void setDefaultRequestProperties(Hashtable properties)
	{
		mDefaultRequestProperties = properties;
	}

	/**
	 * Get the current request header properties. A String-to-String map of
	 * header keys and values, excluding proxy items, cookies and URL
	 * authorization.
	 * 
	 * @return The request header properties for this connection manager.
	 */
	public Hashtable getRequestProperties()
	{
		return (mRequestProperties);
	}

	/**
	 * Set the current request properties. Replaces the current set of fixed
	 * request properties with the given set. This does not replace the
	 * Proxy-Authorization property which is constructed from the values of
	 * {@link #setProxyUser} and {@link #setProxyPassword} values or the
	 * Authorization property which is constructed from the {@link #setUser} and
	 * {@link #setPassword} values. Nor does it replace the Cookie property
	 * which is constructed from the current cookie jar.
	 * 
	 * @param properties
	 *            The new fixed properties.
	 */
	public void setRequestProperties(Hashtable properties)
	{
		mRequestProperties = properties;
	}

	/**
	 * Get the proxy host name, if any.
	 * 
	 * @return Returns the proxy host.
	 */
	public String getProxyHost()
	{
		return (mProxyHost);
	}

	/**
	 * Set the proxy host to use.
	 * 
	 * @param host
	 *            The host to use for proxy access.
	 *            <em>Note: You must also set the proxy {@link #setProxyPort port}.</em>
	 */
	public void setProxyHost(String host)
	{
		mProxyHost = host;
	}

	/**
	 * Get the proxy port number.
	 * 
	 * @return Returns the proxy port.
	 */
	public int getProxyPort()
	{
		return (mProxyPort);
	}

	/**
	 * Set the proxy port number.
	 * 
	 * @param port
	 *            The proxy port.
	 *            <em>Note: You must also set the proxy {@link #setProxyHost host}.</em>
	 */
	public void setProxyPort(int port)
	{
		mProxyPort = port;
	}

	/**
	 * Get the user name for proxy authorization, if any.
	 * 
	 * @return Returns the proxy user, or <code>null</code> if no proxy
	 *         authorization is required.
	 */
	public String getProxyUser()
	{
		return (mProxyUser);
	}

	/**
	 * Set the user name for proxy authorization.
	 * 
	 * @param user
	 *            The proxy user name.
	 *            <em>Note: You must also set the proxy {@link #setProxyPassword password}.</em>
	 */
	public void setProxyUser(String user)
	{
		mProxyUser = user;
	}

	/**
	 * Set the proxy user's password.
	 * 
	 * @return Returns the proxy password.
	 */
	public String getProxyPassword()
	{
		return (mProxyPassword);
	}

	/**
	 * Get the proxy user's password.
	 * 
	 * @param password
	 *            The password for the proxy user.
	 *            <em>Note: You must also set the proxy {@link #setProxyUser user}.</em>
	 */
	public void setProxyPassword(String password)
	{
		mProxyPassword = password;
	}

	/**
	 * Get the user name to access the URL.
	 * 
	 * @return Returns the username that will be used to access the URL, or
	 *         <code>null</code> if no authorization is required.
	 */
	public String getUser()
	{
		return (mUser);
	}

	/**
	 * Set the user name to access the URL.
	 * 
	 * @param user
	 *            The user name for accessing the URL.
	 *            <em>Note: You must also set the {@link #setPassword password}.</em>
	 */
	public void setUser(String user)
	{
		mUser = user;
	}

	/**
	 * Get the URL users's password.
	 * 
	 * @return Returns the URL password.
	 */
	public String getPassword()
	{
		return (mPassword);
	}

	/**
	 * Set the URL users's password.
	 * 
	 * @param password
	 *            The password for the URL.
	 */
	public void setPassword(String password)
	{
		mPassword = password;
	}

	/**
	 * Predicate to determine if cookie processing is currently enabled.
	 * 
	 * @return <code>true</code> if cookies are being processed.
	 */
	public boolean getCookieProcessingEnabled()
	{
		return (null != mCookieJar);
	}

	/**
	 * Enables and disabled cookie processing.
	 * 
	 * @param enable
	 *            if <code>true</code> cookie processing will occur, else cookie
	 *            processing will be turned off.
	 */
	public void setCookieProcessingEnabled(boolean enable)
	{
		if (enable)
			mCookieJar = (null == mCookieJar) ? new Hashtable() : mCookieJar;
		else
			mCookieJar = null;
	}

	/**
	 * Adds a cookie to the cookie jar.
	 * 
	 * @param cookie
	 *            The cookie to add.
	 * @param domain
	 *            The domain to use in case the cookie has no domain attribute.
	 */
	public void setCookie(Cookie cookie, String domain)
	{
		String path;
		Vector cookies;
		Cookie probe;
		boolean found; // flag if a cookie with current name is already there

		if (null != cookie.getDomain()) domain = cookie.getDomain();
		path = cookie.getPath();
		if (null == mCookieJar) mCookieJar = new Hashtable(); // turn on cookie
																// processing
		cookies = (Vector) mCookieJar.get(domain);
		if (null != cookies)
		{
			found = false;
			for (int j = 0; j < cookies.size(); j++)
			{
				probe = (Cookie) cookies.elementAt(j);
				if (probe.getName().equalsIgnoreCase(cookie.getName()))
				{
					// we keep paths sorted most specific to least
					if (probe.getPath().equals(path))
					{
						cookies.setElementAt(cookie, j); // replace
						found = true; // cookie found, set flag
						break;
					}
					else if (path.startsWith(probe.getPath()))
					{
						cookies.insertElementAt(cookie, j);
						found = true; // cookie found, set flag
						break;
					}
				}
			}
			if (!found)
			// there's no cookie with the current name, therefore it's added
			// at the end of the list (faster then inserting at the front)
				cookies.addElement(cookie);
		}
		else
		{ // new cookie list needed
			cookies = new Vector();
			cookies.addElement(cookie);
			mCookieJar.put(domain, cookies);
		}
	}

	/**
	 * Get the monitoring object, if any.
	 * 
	 * @return Returns the monitor, or null if none has been assigned.
	 */
	public ConnectionMonitor getMonitor()
	{
		return (mMonitor);
	}

	/**
	 * Set the monitoring object.
	 * 
	 * @param monitor
	 *            The monitor to set.
	 */
	public void setMonitor(ConnectionMonitor monitor)
	{
		mMonitor = monitor;
	}

	/**
	 * Predicate to determine if url redirection processing is currently
	 * enabled.
	 * 
	 * @return <code>true</code> if redirection is being processed manually.
	 * @see #setRedirectionProcessingEnabled
	 */
	public boolean getRedirectionProcessingEnabled()
	{
		return (mRedirectionProcessingEnabled);
	}

	/**
	 * Enables or disables manual redirection handling. Normally the
	 * <code>HttpURLConnection</code> follows redirections (HTTP response code
	 * 3xx) automatically if the <code>followRedirects</code> property is
	 * <code>true</code>. With this flag set the <code>ConnectionMonitor</code>
	 * performs the redirection processing; The advantage being that cookies (if
	 * enabled) are passed in subsequent requests.
	 * 
	 * @param enabled
	 *            The new state of the redirectionProcessingEnabled property.
	 */
	public void setRedirectionProcessingEnabled(boolean enabled)
	{
		mRedirectionProcessingEnabled = enabled;
	}

	/**
	 * Get the Location field if any.
	 * 
	 * @param http
	 *            The connection to get the location from.
	 */
	protected String getLocation(HttpURLConnection http)
	{
		String key;
		String value;
		String ret;

		ret = null;

		for (int i = 0; ((null == ret) && (null != (value = http.getHeaderField(i)))); i++)
			if ((null != (key = http.getHeaderFieldKey(i))) && (key.equalsIgnoreCase("Location")))
				ret = value;

		return (ret);
	}

	/**
	 * Opens a connection using the given url.
	 * 
	 * @param url
	 *            The url to open.
	 * @return The connection.
	 * @exception ParserException
	 *                if an i/o exception occurs accessing the url.
	 */
	public URLConnection openConnection(URL url) throws ParserException
	{
		boolean repeat;
		int repeated;
		Properties sysprops;
		Hashtable properties;
		Enumeration enumeration;
		String key;
		String value;
		String set = null; // old proxySet value
		String host = null; // old proxyHost value
		String port = null; // old proxyPort value
		String host2 = null; // old http.proxyHost value
		String port2 = null; // old http.proxyPort value
		HttpURLConnection http;
		String auth;
		String encoded;
		int code;
		String uri;
		URLConnection ret;

		repeated = 0;
		do
		{
			repeat = false;
			try
			{
				try
				{
					// set up for proxy
					if ((null != getProxyHost()) && (0 != getProxyPort()))
					{
						sysprops = System.getProperties();
						set = (String) sysprops.put("proxySet", "true");
						host = (String) sysprops.put("proxyHost", getProxyHost());
						port = (String) sysprops.put("proxyPort", Integer.toString(getProxyPort()));
						// see
						// http://java.sun.com/j2se/1.4.2/docs/guide/net/properties.html
						host2 = (String) sysprops.put("http.proxyHost", getProxyHost());
						port2 = (String) sysprops.put("http.proxyPort",
								Integer.toString(getProxyPort()));
						System.setProperties(sysprops);

					}

					// open the connection... but don't connect yet
					ret = url.openConnection();
					if (ret instanceof HttpURLConnection)
					{
						http = (HttpURLConnection) ret;

						if (getRedirectionProcessingEnabled())
							http.setInstanceFollowRedirects(false);

						// set the fixed request properties
						properties = getRequestProperties();
						if (null != properties)
							for (enumeration = properties.keys(); enumeration.hasMoreElements();)
							{
								key = (String) enumeration.nextElement();
								value = (String) properties.get(key);
								ret.setRequestProperty(key, value);
							}

						// set the proxy name and password
						if ((null != getProxyUser()) && (null != getProxyPassword()))
						{
							auth = getProxyUser() + ":" + getProxyPassword();
							encoded = encode(auth.getBytes("ISO-8859-1"));
							ret.setRequestProperty("Proxy-Authorization", encoded);
						}

						// set the URL name and password
						if ((null != getUser()) && (null != getPassword()))
						{
							auth = getUser() + ":" + getPassword();
							encoded = encode(auth.getBytes("ISO-8859-1"));
							ret.setRequestProperty("Authorization", "Basic " + encoded);
						}

						if (getCookieProcessingEnabled())
						// set the cookies based on the url
							addCookies(ret);

						if (null != getMonitor()) getMonitor().preConnect(http);
					}
					else
						http = null;

					try
					{
						ret.connect();

						if (null != http)
						{
							if (null != getMonitor()) getMonitor().postConnect(http);

							if (getCookieProcessingEnabled()) parseCookies(ret);

							code = http.getResponseCode();
							if ((3 == (code / 100)) && (repeated < 20))
								if (null != (uri = getLocation(http)))
								{
									url = new URL(uri);
									repeat = true;
									repeated++;
								}
						}
					}
					catch (UnknownHostException uhe)
					{
						int message = (int) (Math.random() * FOUR_OH_FOUR.length);
						throw new ParserException(FOUR_OH_FOUR[message], uhe);
					}
					catch (IOException ioe)
					{
						throw new ParserException(ioe.getMessage(), ioe);
					}
				}
				finally
				{
					if ((null != getProxyHost()) && (0 != getProxyPort()))
					{
						sysprops = System.getProperties();
						if (null != set)
							sysprops.put("proxySet", set);
						else
							sysprops.remove("proxySet");
						if (null != host)
							sysprops.put("proxyHost", host);
						else
							sysprops.remove("proxyHost");
						if (null != port)
							sysprops.put("proxyPort", port);
						else
							sysprops.remove("proxyPort");
						if (null != host2)
							sysprops.put("http.proxyHost", host2);
						else
							sysprops.remove("http.proxyHost");
						if (null != port2)
							sysprops.put("http.proxyPort", port2);
						else
							sysprops.remove("http.proxyPort");
						System.setProperties(sysprops);
					}
				}
			}
			catch (IOException ioe)
			{
				String msg = "Error in opening a connection to " + url.toExternalForm();
				ParserException ex = new ParserException(msg, ioe);
				throw ex;
			}
		} while (repeat);

		return (ret);
	}

	/**
	 * Encodes a byte array into BASE64 in accordance with <a
	 * href="http://www.faqs.org/rfcs/rfc2045.html">RFC 2045</a>.
	 * 
	 * @param array
	 *            The bytes to convert.
	 * @return A BASE64 encoded string.
	 */
	public final static String encode(byte[] array)
	{
		int last; // last byte
		int count; // character count
		int separators; // line separator count
		int length; // length of returned string
		char[] encoded; // encoded characters
		int left; // bytes left
		int end;
		int block; // encoding buffer
		int r; // shift count
		int n; // byte to encode
		int index; // index into output array
		String ret;

		if ((null != array) && (0 != array.length))
		{
			last = array.length - 1;
			count = (last / 3 + 1) << 2;
			separators = (count - 1) / 76;
			length = count + separators;
			encoded = new char[length];
			index = 0;
			separators = 0;
			for (int i = 0; i <= last; i += 3)
			{
				left = last - i;
				end = (left > 1 ? 2 : left);

				// collect 1 to 3 bytes to encode
				block = 0;
				r = 16;
				for (int j = 0; j <= end; j++)
				{
					n = array[i + j];
					block += (n < 0 ? n + 256 : n) << r;
					r -= 8;
				}

				// encode into 2-4 chars padding with '=' if no data left
				encoded[index++] = BASE64_CHAR_TABLE[(block >>> 18) & 0x3f];
				encoded[index++] = BASE64_CHAR_TABLE[(block >>> 12) & 0x3f];
				encoded[index++] = left > 0 ? BASE64_CHAR_TABLE[(block >>> 6) & 0x3f] : '=';
				encoded[index++] = left > 1 ? BASE64_CHAR_TABLE[block & 0x3f] : '=';

				if ((0 == (index - separators) % 76) && (index < length))
				{
					encoded[index++] = '\n';
					separators += 1;
				}
			}
			ret = new String(encoded);
		}
		else
			ret = "";

		return (ret);
	}

	/**
	 * Turn spaces into %20. ToDo: make this more generic (see RFE #1010593
	 * provide URL encoding/decoding utilities).
	 * 
	 * @param url
	 *            The url containing spaces.
	 * @return The URL with spaces as %20 sequences.
	 */
	public String fixSpaces(String url)
	{
		int index;
		int length;
		char ch;
		StringBuffer buffer;

		index = url.indexOf(' ');
		if (-1 != index)
		{
			length = url.length();
			buffer = new StringBuffer(length * 3);
			buffer.append(url.substring(0, index));
			for (int i = index; i < length; i++)
			{
				ch = url.charAt(i);
				if (ch == ' ')
					buffer.append("%20");
				else
					buffer.append(ch);
			}
			url = buffer.toString();
		}

		return (url);
	}

	/**
	 * Opens a connection based on a given string. The string is either a file,
	 * in which case <code>file://localhost</code> is prepended to a canonical
	 * path derived from the string, or a url that begins with one of the known
	 * protocol strings, i.e. <code>http://</code>. Embedded spaces are silently
	 * converted to %20 sequences.
	 * 
	 * @param string
	 *            The name of a file or a url.
	 * @return The connection.
	 * @exception ParserException
	 *                if the string is not a valid url or file.
	 */
	public URLConnection openConnection(String string) throws ParserException
	{
		final String prefix = "file://localhost";
		String resource;
		URL url;
		StringBuffer buffer;
		URLConnection ret;

		try
		{
			url = new URL(fixSpaces(string));
			ret = openConnection(url);
		}
		catch (MalformedURLException murle)
		{ // try it as a file
			try
			{
				File file = new File(string);
				resource = file.getCanonicalPath();
				buffer = new StringBuffer(prefix.length() + resource.length());
				buffer.append(prefix);
				if (!resource.startsWith("/")) buffer.append("/");
				buffer.append(resource);
				url = new URL(fixSpaces(buffer.toString()));
				ret = openConnection(url);
			}
			catch (MalformedURLException murle2)
			{
				String msg = "Error in opening a connection to " + string;
				ParserException ex = new ParserException(msg, murle2);
				throw ex;
			}
			catch (IOException ioe)
			{
				String msg = "Error in opening a connection to " + string;
				ParserException ex = new ParserException(msg, ioe);
				throw ex;
			}
		}

		return (ret);
	}

	/**
	 * Generate a HTTP cookie header value string from the cookie jar.
	 * 
	 * <pre>
	 *   The syntax for the header is:
	 * 
	 *    cookie          =       "Cookie:" cookie-version
	 *                            1*((";" | ",") cookie-value)
	 *    cookie-value    =       NAME "=" VALUE [";" path] [";" domain]
	 *    cookie-version  =       "$Version" "=" value
	 *    NAME            =       attr
	 *    VALUE           =       value
	 *    path            =       "$Path" "=" value
	 *    domain          =       "$Domain" "=" value
	 * 
	 * </pre>
	 * 
	 * @param connection
	 *            The connection being accessed.
	 * @see <a href="http://www.ietf.org/rfc/rfc2109.txt">RFC 2109</a>
	 * @see <a href="http://www.ietf.org/rfc/rfc2396.txt">RFC 2396</a>
	 */
	public void addCookies(URLConnection connection)
	{
		Vector list;
		URL url;
		String host;
		String path;
		String domain;

		if (null != mCookieJar)
		{
			list = null;
			// get the site from the URL
			url = connection.getURL();
			host = url.getHost();
			path = url.getPath();
			if (0 == path.length()) path = "/";
			if (null != host)
			{ // http://www.objectsdevelopment.com/portal/modules/freecontent/content/javawebserver.html
				list = addCookies((Vector) mCookieJar.get(host), path, list);
				domain = getDomain(host);
				if (null != domain)
					list = addCookies((Vector) mCookieJar.get(domain), path, list);
				else
					// maybe it is the domain we're accessing
					list = addCookies((Vector) mCookieJar.get("." + host), path, list);
			}
			if (null != list)
				connection.setRequestProperty("Cookie", generateCookieProperty(list));
		}
	}

	/**
	 * Add qualified cookies from cookies into list.
	 * 
	 * @param cookies
	 *            The list of cookies to check (may be null).
	 * @param path
	 *            The path being accessed.
	 * @param list
	 *            The list of qualified cookies.
	 * @return The list of qualified cookies.
	 */
	protected Vector addCookies(Vector cookies, String path, Vector list)
	{
		Cookie cookie;
		Date expires;
		Date now;

		if (null != cookies)
		{
			now = new Date();
			for (int i = 0; i < cookies.size(); i++)
			{
				cookie = (Cookie) cookies.elementAt(i);
				expires = cookie.getExpiryDate();
				if ((null != expires) && expires.before(now))
				{
					cookies.remove(i);
					i--; // dick with the loop variable
				}
				else if (path.startsWith(cookie.getPath()))
				{
					if (null == list) list = new Vector();
					list.addElement(cookie);
				}
			}
		}

		return (list);
	}

	/**
	 * Get the domain from a host.
	 * 
	 * @param host
	 *            The supposed host name.
	 * @return The domain (with the leading dot), or null if the domain cannot
	 *         be determined.
	 */
	protected String getDomain(String host)
	{
		StringTokenizer tokenizer;
		int count;
		String server;
		int length;
		boolean ok;
		char c;
		String ret;

		ret = null;

		tokenizer = new StringTokenizer(host, ".");
		count = tokenizer.countTokens();
		if (3 <= count)
		{
			// have at least two dots,
			// check if we were handed an IP address by mistake
			length = host.length();
			ok = false;
			for (int i = 0; i < length && !ok; i++)
			{
				c = host.charAt(i);
				if (!(Character.isDigit(c) || (c == '.'))) ok = true;
			}
			if (ok)
			{
				// so take everything after the first token
				server = tokenizer.nextToken();
				length = server.length();
				ret = host.substring(length);
			}
		}

		return (ret);
	}

	/**
	 * Creates the cookie request property value from the list of valid cookies
	 * for the domain.
	 * 
	 * @param cookies
	 *            The list of valid cookies to be encoded in the request.
	 * @return A string suitable for inclusion as the value of the "Cookie:"
	 *         request property.
	 */
	protected String generateCookieProperty(Vector cookies)
	{
		int version;
		Cookie cookie;
		StringBuffer buffer;
		String ret;

		ret = null;

		buffer = new StringBuffer();
		version = 0;
		for (int i = 0; i < cookies.size(); i++)
			version = Math.max(version, ((Cookie) cookies.elementAt(i)).getVersion());
		if (0 != version)
		{
			buffer.append("$Version=\"");
			buffer.append(version);
			buffer.append("\"");
		}
		for (int i = 0; i < cookies.size(); i++)
		{
			cookie = (Cookie) cookies.elementAt(i);
			if (0 != buffer.length()) buffer.append("; ");
			buffer.append(cookie.getName());
			buffer.append(cookie.getName().equals("") ? "" : "=");
			if (0 != version) buffer.append("\"");
			buffer.append(cookie.getValue());
			if (0 != version) buffer.append("\"");
			if (0 != version)
			{
				if ((null != cookie.getPath()) && (0 != cookie.getPath().length()))
				{
					buffer.append("; $Path=\"");
					buffer.append(cookie.getPath());
					buffer.append("\"");
				}
				if ((null != cookie.getDomain()) && (0 != cookie.getDomain().length()))
				{
					buffer.append("; $Domain=\"");
					buffer.append(cookie.getDomain());
					buffer.append("\"");
				}
			}
		}
		if (0 != buffer.length()) ret = buffer.toString();

		return (ret);
	}

	/**
	 * Check for cookie and parse into cookie jar.
	 * 
	 * @param connection
	 *            The connection to extract cookie information from.
	 */
	public void parseCookies(URLConnection connection)
	{
		String string;
		Vector cookies;
		StringTokenizer tokenizer;
		String token;
		int index;
		String name;
		String key;
		String value;
		Cookie cookie;

		string = connection.getHeaderField("Set-Cookie");
		if (null != string)
		{
			// set-cookie = "Set-Cookie:" cookies
			// cookies = 1#cookie
			// cookie = NAME "=" VALUE *(";" cookie-av)
			// NAME = attr
			// VALUE = value
			// cookie-av = "Comment" "=" value
			// | "Domain" "=" value
			// | "Max-Age" "=" value
			// | "Path" "=" value
			// | "Secure"
			// | "Version" "=" 1*DIGIT
			cookies = new Vector();
			tokenizer = new StringTokenizer(string, ";,", true);
			cookie = null;
			while (tokenizer.hasMoreTokens())
			{
				token = tokenizer.nextToken().trim();
				if (token.equals(";"))
					continue;
				else if (token.equals(","))
				{
					cookie = null;
					continue;
				}

				index = token.indexOf('=');
				if (-1 == index)
				{
					if (null == cookie)
					{ // an unnamed cookie
						name = "";
						value = token;
						key = name;
					}
					else
					{
						name = token;
						value = null;
						key = name.toLowerCase();
					}
				}
				else
				{
					name = token.substring(0, index);
					value = token.substring(index + 1);
					key = name.toLowerCase();
				}

				if (null == cookie)
				{
					try
					{
						cookie = new Cookie(name, value);
						cookies.addElement(cookie);
					}
					catch (IllegalArgumentException iae)
					{
						// should print a warning
						// for now just bail
						break;
					}
				}
				else
				{
					if (key.equals("expires")) // Wdy, DD-Mon-YY HH:MM:SS GMT
					{
						String comma = tokenizer.nextToken();
						String rest = tokenizer.nextToken();
						try
						{
							Date date = mFormat.parse(value + comma + rest);
							cookie.setExpiryDate(date);
						}
						catch (ParseException pe)
						{
							// ok now what
							cookie.setExpiryDate(null);
						}
					}
					else if (key.equals("domain"))
						cookie.setDomain(value);
					else if (key.equals("path"))
						cookie.setPath(value);
					else if (key.equals("secure"))
						cookie.setSecure(true);
					else if (key.equals("comment"))
						cookie.setComment(value);
					else if (key.equals("version"))
						cookie.setVersion(Integer.parseInt(value));
					else if (key.equals("max-age"))
					{
						Date date = new Date();
						long then = date.getTime() + Integer.parseInt(value) * 1000;
						date.setTime(then);
						cookie.setExpiryDate(date);
					}
					else
						// error,? unknown attribute,
						// maybe just another cookie
						// not separated by a comma
						try
						{
							cookie = new Cookie(name, value);
							cookies.addElement(cookie);
						}
						catch (IllegalArgumentException iae)
						{
							// should print a warning
							// for now just bail
							break;
						}
				}
			}
			if (0 != cookies.size()) saveCookies(cookies, connection);
		}
	}

	/**
	 * Save the cookies received in the response header.
	 * 
	 * @param list
	 *            The list of cookies extracted from the response header.
	 * @param connection
	 *            The connection (used when a cookie has no domain).
	 */
	protected void saveCookies(Vector list, URLConnection connection)
	{
		Cookie cookie;
		String domain;

		for (int i = 0; i < list.size(); i++)
		{
			cookie = (Cookie) list.elementAt(i);
			domain = cookie.getDomain();
			if (null == domain) domain = connection.getURL().getHost();
			setCookie(cookie, domain);
		}
	}
}
